# PoP - Policing Socio-Geographic Boundaries and Inequality
# script for creating final dataset for Atlanta.

#### libraries ####

library(MASS)
library(sp)
library(stargazer)
library(sf)
library(matrixStats)
library(tidyverse)
library(scales)
library(tidycensus)
library(gridExtra)
library(spdep)
library(grid)
library(ggplot2)
library(lattice)


#### downloading census data #### 

# Download Decennial Census data 2010

census_api_key(key = 'db6400c6d9d5b405e0f7f1e8872c3b8299afa6fb', install = TRUE)

vars_census <- c("P001001", "P012006", "P012007", "P012008", "P012009", "P012010",
                 "P012011", "P012012", "P005003", "P005004", "P005006", "P005010",
                 "P015001", "P005005", "P005007", "P005008", "P005009")

atl_sf1 <- tidycensus::get_decennial(geography = "block", state = "GA",
                                     county = "Fulton County", variables = vars_census,
                                     year = 2010, output = "wide", geometry = TRUE)

# Download ACS data 2011-2016
# mhhi
# % poverty
# % unemployed
# college


vars_acs <- c("C24010_004E", "C24010_040E", "C24010_001E", "B23025_005E", "B23025_003E",
              "B17021_001E", "B17021_002E", "B15002_011E", "B15002_028E", "B15002_001E",
              "B11003_016E", "B11001_001E", "B25001_001E", "B25003_003E", "B25002_002E",
              "B25002_003E", "B16004_001E", "B16004_006E", "B16004_007E", "B16004_008E",
              "B16004_011E", "B16004_012E", "B16004_013E", "B16004_016E", "B16004_017E",
              "B16004_018E", "B16004_021E", "B16004_022E", "B16004_023E", "B16004_028E",
              "B16004_029E", "B16004_030E", "B16004_033E", "B16004_034E", "B16004_035E",
              "B16004_038E", "B16004_039E", "B16004_040E", "B16004_043E", "B16004_044E",
              "B16004_045E", "B16004_050E", "B16004_051E", "B16004_052E", "B16004_055E",
              "B16004_056E", "B16004_057E", "B16004_060E", "B16004_061E", "B16004_062E",
              "B16004_065E", "B16004_066E", "B16004_067E", "B16004_008E", "B16004_013E",
              "B16004_018E", "B16004_023E", "B16004_030E", "B16004_035E", "B16004_040E",
              "B16004_045E", "B16004_052E", "B16004_057E", "B16004_062E", "B16004_067E",
              "B16004_001E", "B25038_004E", "B25038_011E", "B25038_004E", "B25038_011E",
              "B19013_001", # mhhi 
              "B15003_001", # education universe 
              "B15003_022", "B15003_023", "B15003_024", "B15003_025", # number of college educated or more
              "B25003_001", # homeowner universe
              "B25003_002") # homeowner

vars_loaded = load_variables(dataset = "acs5", year = 2013)
# View(vars_loaded)

atl_acs <- tidycensus::get_acs(geography = "block group", state = "GA",
                               county = "Fulton County", variables = vars_acs,
                               year = 2013, output = "wide", geometry = TRUE)

# herfindahl or fractionalization index

f_index <- function(x) 1 - rowSums(x^2)

# factor analysis

fac <- function(x, factors, rotation = "varimax", scores = "regression") {
  ok <- complete.cases(x)
  fa <- factanal(x[ok,], factors, rotation=rotation, scores=scores)
  score <- rep(NA, fa$n.obs + sum(!ok))
  score[ok] <- fa$scores
  return(score)
}


# recode Decennial Census

atl_sf1 <- atl_sf1 %>%
  transmute(
    BLK_CODE = GEOID,
    pop = P001001,
    hh = P015001,
    race_white = P005003,
    race_black = P005004,
    race_asian = P005006,
    race_hisp = P005010,
    race_other = P005005 + P005007 + P005008 + P005009,
    p_race_white = race_white / pop,
    p_race_black = race_black / pop,
    p_race_asian = race_asian / pop,
    p_race_hisp = race_hisp / pop,
    p_race_other = race_other / pop,
    age_15_35_male = P012006 + P012007 + P012008 + P012009 + P012010 +
      P012011 + P012012,
    hhi = f_index(cbind(p_race_white, p_race_black, p_race_asian,
                        p_race_hisp, p_race_other)),
    geometry
  )

atl_sf1$GEOID = substring(atl_sf1$BLK_CODE, 1, 12)

# recode ACS

atl_acs <- atl_acs %>%
  transmute(
    BG_CODE = GEOID,
    occ_management = C24010_004E + C24010_040E,
    occ_universe = C24010_001E,
    emp_unemployed = B23025_005E,
    emp_civ_lab_force = B23025_003E,
    poverty_universe = B17021_001E,
    poverty = B17021_002E,
    edu_hs = B15002_011E + B15002_028E,
    edu_universe = B15002_001E,
    hh_single_mother = B11003_016E,
    mhhi = B19013_001E, 
    hh = B11001_001E,
    hu = B25001_001E,
    hu_occupied_renter = B25003_003E,
    hu_occupied = B25002_002E,
    hu_vacant = B25002_003E,
    lang_universe = B16004_001E,
    lang_eng_limited = # Speak Eng less than "very well"
      B16004_006E + B16004_007E + B16004_008E +
      B16004_011E + B16004_012E + B16004_013E +
      B16004_016E + B16004_017E + B16004_018E +
      B16004_021E + B16004_022E + B16004_023E +
      B16004_028E + B16004_029E + B16004_030E +
      B16004_033E + B16004_034E + B16004_035E +
      B16004_038E + B16004_039E + B16004_040E +
      B16004_043E + B16004_044E + B16004_045E +
      B16004_050E + B16004_051E + B16004_052E +
      B16004_055E + B16004_056E + B16004_057E +
      B16004_060E + B16004_061E + B16004_062E +
      B16004_065E + B16004_066E + B16004_067E,
    lang_eng_no = B16004_008E + B16004_013E + B16004_018E + B16004_023E +
      B16004_030E + B16004_035E + B16004_040E + B16004_045E +
      B16004_052E + B16004_057E + B16004_062E + B16004_067E,
    lang_universe = B16004_001E,
    housing_moved_2000_2009 = B25038_004E + B25038_011E,
    housing_moved_2000_2009 = B25038_004E + B25038_011E,
    p_occ_management = occ_management / occ_universe,
    p_emp_unemployed = emp_unemployed / emp_civ_lab_force,
    p_poverty = poverty / poverty_universe,
    p_edu_hs = edu_hs / edu_universe,
    p_single_mother = hh_single_mother / hh,
    p_hu_occupied_renter = hu_occupied_renter / hu_occupied,
    p_hu_vacant = hu_vacant / hu,
    p_lang_eng_limited = lang_eng_limited / lang_universe,
    p_lang_eng_no = lang_eng_no / lang_universe,
    pcol = (B15003_022E + B15003_023E + B15003_024E + B15003_025E) / B15003_001E,
    pown = B25003_002E / B25003_001E,
    p_housing_moved_2000_2009 = housing_moved_2000_2009 / hu_occupied,
    # concentrated disadvantage, residential instability and immigrant concentration
    con_disadv = fac(cbind(p_occ_management, p_emp_unemployed, p_poverty,
                           p_edu_hs, p_single_mother), factors=1),
    res_instab = fac(cbind(p_hu_occupied_renter, p_housing_moved_2000_2009,
                           p_hu_vacant), factors=1),
    immi_con = rowMeans(cbind(p_lang_eng_limited, p_lang_eng_no))
  ) %>% dplyr:: select(BG_CODE, con_disadv, res_instab, immi_con, mhhi, pcol, pown, p_poverty,
         p_emp_unemployed)

# join decennial census with ACS data

atl_sf1$BG_CODE = substring(atl_sf1$BLK_CODE, 1, 12)
atl_bg <- atl_sf1 %>% left_join((atl_acs %>% mutate(geometry = NULL) %>% as.data.frame), by = "BG_CODE")

# merg with boundary data

atl_shp = read_sf("Official_City_Boundary.shp") %>% filter(NAME == "Atlanta") %>% dplyr::select(geometry)

atl_shp = st_transform(atl_shp, st_crs(atl_bg))
st_crs(atl_shp) = st_crs(atl_bg)

atl_bg = st_make_valid(atl_bg, dist = 0)
atl_shp = st_make_valid(atl_shp, dist = 0)
atl_bg = st_intersection(atl_shp, atl_bg)

# wombling, generating difference measure

atl_bg = st_make_valid(atl_bg, dist=0)

uq_bgs = unique(atl_bg$BLK_CODE)

test = poly2nb(atl_bg)
atl_bg$p_race_black_blv = NA
atl_bg$p_race_white_blv = NA
atl_bg$p_race_hisp_blv = NA
atl_bg$p_race_asian_blv = NA

# loop to generate measure

for (i in 1:length(uq_bgs)) {
  
  print(paste0("Iteration ", i))
  
  atl_bg$p_race_black_blv[i] = 
    max(abs(atl_bg[atl_bg$BLK_CODE %in% uq_bgs[i], ]$p_race_black - 
              atl_bg[test[[i]], ]$p_race_black), na.rm = TRUE)
  atl_bg$p_race_white_blv[i] = 
    max(abs(atl_bg[atl_bg$BLK_CODE %in% uq_bgs[i], ]$p_race_white - 
              atl_bg[test[[i]], ]$p_race_white), na.rm = TRUE)
  atl_bg$p_race_hisp_blv[i] = 
    max(abs(atl_bg[atl_bg$BLK_CODE %in% uq_bgs[i], ]$p_race_hisp - 
              atl_bg[test[[i]], ]$p_race_hisp), na.rm = TRUE)
  atl_bg$p_race_asian_blv[i] = 
    max(abs(atl_bg[atl_bg$BLK_CODE %in% uq_bgs[i], ]$p_race_asian - 
              atl_bg[test[[i]], ]$p_race_asian), na.rm = TRUE)
  
}


# read in categorized 311 data
load("atl_311_cat.RData")

# convert to sf file
atl_311_sf <- atl_311_cat %>% mutate(long_x = ifelse(latitude < 0,latitude,longitude),
                                    lat_y = ifelse(longitude > 0,longitude,latitude))
summary(atl_311_sf$long_x)
summary(atl_311_sf$lat_y)
# remove missing values
atl_311_sf = atl_311_sf %>% filter(!is.na(latitude))
atl_311_sf  <- st_as_sf(x = atl_311_sf, coords = c(x="long_x",y="lat_y"))


# set crs btw census data and 311 calls
st_crs(atl_311_sf) = st_crs(atl_bg)
atl_311_sf = st_transform(atl_311_sf, st_crs(atl_bg))

# locate 311 calls with census blocks
atl_311_merged <- st_join(atl_311_sf,atl_bg,
                            join = st_intersects)

atl_311_merged = atl_311_merged %>% dplyr::select(`Service Request #`,BLK_CODE, call_code)


# convert call_code to character
atl_311_merged$call_code <- as.character(atl_311_merged$call_code)

# reshape data from long form to wide form with each call code as column
atl_311_merged$val = 1
wide_311_calls = atl_311_merged %>% 
  tidyr::pivot_wider(names_from = "call_code", values_from = "val", values_fill = 0)

## sum number of each type of call per category by BG and year
wide_311_calls_sum = wide_311_calls %>% 
  group_by(BLK_CODE) %>% 
  summarise(across(c("16","NA","13","9","15","5","6","7","8","11","17","10","12","1","2","4","3","14"), sum)) %>% 
  as.data.frame %>% mutate(geometry = NULL)

# sum total calls per year
wide_311_calls_sum = wide_311_calls_sum %>%
  mutate(total_calls = rowSums(across(2:19)))

# sum top 4 categories 
wide_311_calls_sum$high_SES <- wide_311_calls_sum$`1` + wide_311_calls_sum$`9` + wide_311_calls_sum$`16` + wide_311_calls_sum$`5`
wide_311_calls_sum$high_SES_stan <- (wide_311_calls_sum$high_SES/wide_311_calls_sum$total_calls)

atl_311_final = wide_311_calls_sum

# merge with atl_bg based on BLK_CODE
atl_311_final = atl_311_final %>% dplyr::select(c(BLK_CODE,high_SES_stan))
atl_bg <- merge(atl_bg, atl_311_final, by = "BLK_CODE", all.x = TRUE)

# wombling, generating diff measure between proportion of high SES calls
atl_bg = st_make_valid(atl_bg, dist=0)

uq_bgs = unique(atl_bg$BLK_CODE)

test = poly2nb(atl_bg)
atl_bg$ses_blv = NA

# loop to generate measure

for (i in 1:length(uq_bgs)) {
  
  print(paste0("Iteration ", i))
  
  atl_bg$ses_blv[i] = 
    max(abs(atl_bg[atl_bg$BLK_CODE %in% uq_bgs[i], ]$high_SES_stan - 
              atl_bg[test[[i]], ]$high_SES_stan), na.rm = TRUE)
}

atl_bg$ses_blv = 
  ifelse(atl_bg$ses_blv == Inf | atl_bg$ses_blv == -Inf, NA, atl_bg$ses_blv)

# save atl with boundary measures
save(x = atl_bg, file = "atl_blv.RData")

### data is confidential but providing the code used to aggregate geolocated arrests by census block
### at the end of the code that is commented out, load the dataset with arrests to proceed in the script


## Merge wombling block level data w arrest data
# Load arrest data
# load("atl_arrests_geo_race.RData") 

# atlanta_geo = atl_arrests_geo_race
# 
# # convert to sf
# atlanta_geo <- subset(atlanta_geo, !is.na(lon))
# atlanta_geo = st_as_sf(atlanta_geo, coords = c("lon", "lat"))
# 
# # figuring out intersection with the shapefile data
# 
# st_crs(atlanta_geo) = st_crs(atl_bg)
# atlanta_geo = st_transform(atlanta_geo, st_crs(atl_bg))
# 
# # specifically, find intersection between arrests by category
# 
# atl_ints = st_intersects(atlanta_geo, atl_bg)
# 
# 
# atl_ints2a = st_intersects(atlanta_geo %>% filter(felony == 1), atl_bg)
# atl_ints2b = st_intersects(atlanta_geo %>% filter(felony == 0), atl_bg)
# atl_ints2c = st_intersects(atlanta_geo %>% filter(violent == 1), atl_bg)
# atl_ints2d = st_intersects(atlanta_geo %>% filter(violent == 0), atl_bg)
# atl_ints2e = st_intersects(atlanta_geo %>% filter(against_society == 1), atl_bg)
# atl_ints2f = st_intersects(atlanta_geo %>% filter(against_person == 1), atl_bg)
# atl_ints2g = st_intersects(atlanta_geo %>% filter(against_property == 1), atl_bg)
# # filter by race of arrestee
# atl_ints2h = st_intersects(atlanta_geo %>% filter(r_blk == 1), atl_bg)
# atl_ints2i = st_intersects(atlanta_geo %>% filter(r_hisp == 1), atl_bg)
# atl_ints2j = st_intersects(atlanta_geo %>% filter(r_wht == 1), atl_bg)
# atl_ints2k = st_intersects(atlanta_geo %>% filter(r_oth == 1), atl_bg)
# 
# 
# theout = atl_bg[atl_ints %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>% 
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(total_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2a = atl_bg[atl_ints2a %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(felony_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2b = atl_bg[atl_ints2b %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(misdemeanor_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2c = atl_bg[atl_ints2c %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(violent_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2d = atl_bg[atl_ints2d %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(nonviolent_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2e = atl_bg[atl_ints2e %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(society_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2f = atl_bg[atl_ints2f %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(person_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2g = atl_bg[atl_ints2g %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(property_arrests = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2h = atl_bg[atl_ints2h %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(arrests_black = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2i = atl_bg[atl_ints2i %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(arrests_hisp = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2j = atl_bg[atl_ints2j %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(arrests_white = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# theout2k = atl_bg[atl_ints2k %>% unlist, ] %>% 
#   dplyr::select(BLK_CODE) %>%
#   mutate(count = 1) %>% 
#   group_by(BLK_CODE) %>% 
#   summarize(arrests_other = sum(count, na.rm = TRUE)) %>% 
#   as.data.frame %>% 
#   mutate(geometry = NULL)
# 
# 
# atl_bg2 = merge(atl_bg, theout, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2a, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2b, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2c, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2d, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2e, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2f, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2g, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2h, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2i, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2j, by = "BLK_CODE", all.x = TRUE)
# atl_bg2 = merge(atl_bg2, theout2k, by = "BLK_CODE", all.x = TRUE)
# 
# 
# atl_bg2$total_arrests = ifelse(is.na(atl_bg2$total_arrests), 0, atl_bg2$total_arrests)
# atl_bg2$felony_arrests = ifelse(is.na(atl_bg2$felony_arrests), 0, atl_bg2$felony_arrests)
# atl_bg2$misdemeanor_arrests = ifelse(is.na(atl_bg2$misdemeanor_arrests), 0, atl_bg2$misdemeanor_arrests)
# atl_bg2$violent_arrests = ifelse(is.na(atl_bg2$violent_arrests), 0, atl_bg2$violent_arrests)
# atl_bg2$nonviolent_arrests = ifelse(is.na(atl_bg2$nonviolent_arrests), 0, atl_bg2$nonviolent_arrests)
# atl_bg2$society_arrests = ifelse(is.na(atl_bg2$society_arrests), 0, atl_bg2$society_arrests)
# atl_bg2$person_arrests = ifelse(is.na(atl_bg2$person_arrests), 0, atl_bg2$person_arrests)
# atl_bg2$property_arrests = ifelse(is.na(atl_bg2$property_arrests), 0, atl_bg2$property_arrests)
# atl_bg2$arrests_black = ifelse(is.na(atl_bg2$arrests_black), 0, atl_bg2$arrests_black)
# atl_bg2$arrests_hisp = ifelse(is.na(atl_bg2$arrests_hisp), 0, atl_bg2$arrests_hisp)
# atl_bg2$arrests_white = ifelse(is.na(atl_bg2$arrests_white), 0, atl_bg2$arrests_white)
# atl_bg2$arrests_other = ifelse(is.na(atl_bg2$arrests_other), 0, atl_bg2$arrests_other)
# 

# read in dataset with aggregate arrests by type to continue

load("atl_block_arrests.RData")

atl_block_arrests <- atl_bg2

#### incorporating crime data ####

atlcrime = read_csv("COBRA-2009-2019.csv")

# so longitude and latitude need to be fixed. 

atlcrime = atlcrime %>%
  filter(!is.na(Longitude))

atlcrime  <- st_as_sf(x = atlcrime, coords = c("Longitude", "Latitude"))

# cleaning crime data 
# subset by relevant years (ATL arrests for years 2012-2014)
atlcrime <- subset(atlcrime, `Occur Date` >= "2012-01-01" & `Occur Date` <= "2014-12-31")

## aggregating crime data by categories
unique(atlcrime$`UCR Literal`)

# add column classifying crime based on UCR description
atlcrime = atlcrime %>% 
  mutate(crime_type = case_when(
    `UCR Literal` == "LARCENY-NON VEHICLE" | `UCR Literal` == "LARCENY-FROM VEHICLE" |
      `UCR Literal` == "ROBBERY-PEDESTRIAN" | `UCR Literal` == "ROBBERY-RESIDENCE" |
      `UCR Literal` == "AUTO THEFT" | `UCR Literal` == "BURGLARY-RESIDENCE" |
      `UCR Literal` == "BURGLARY-NONRES" | `UCR Literal` == "ROBBERY-COMMERCIAL" ~ "Property",
    `UCR Literal` == "AGG ASSAULT" | `UCR Literal` == "HOMICIDE" |
      `UCR Literal` == "MANSLAUGHTER" ~ "Violent",))


# generating 2 categories
# 1) violent crime 
# 2) property crime

atlcrime$crime_violent <- ifelse(atlcrime$crime_type == "Violent", 1, 0)
atlcrime$crime_property <- ifelse(atlcrime$crime_type == "Property", 1, 0)

atlcrime = atlcrime %>% 
  dplyr::select(crime_violent, crime_property, geometry)

# figuring out intersection 
# set CRS system
atlcrime  <- st_set_crs(atlcrime, 4326)
atlcrime <- st_transform(atlcrime, crs = 4326)
atl_bg2 <- st_transform(atl_bg2, crs = 4326)

atl_ints3a = st_intersects(atlcrime %>% filter(crime_violent == 1), atl_bg2)
atl_ints3b = st_intersects(atlcrime %>% filter(crime_property == 1), atl_bg2)

theout3a = atl_bg2[atl_ints3a %>% unlist, ] %>% 
  dplyr::select(BLK_CODE) %>%
  mutate(count = 1) %>% 
  group_by(BLK_CODE) %>% 
  summarize(crime_violent = sum(count, na.rm = TRUE)) %>% 
  as.data.frame %>% 
  mutate(geometry = NULL)

theout3b = atl_bg2[atl_ints3b %>% unlist, ] %>% 
  dplyr::select(BLK_CODE) %>%
  mutate(count = 1) %>% 
  group_by(BLK_CODE) %>% 
  summarize(crime_property = sum(count, na.rm = TRUE)) %>% 
  as.data.frame %>% 
  mutate(geometry = NULL)

atl_bg3 = merge(atl_bg2, theout3a, by = "BLK_CODE", all.x = TRUE)
atl_bg3 = merge(atl_bg3, theout3b, by = "BLK_CODE", all.x = TRUE)

atl_bg3$crime_violent = ifelse(is.na(atl_bg3$crime_violent), 0, atl_bg3$crime_violent)
atl_bg3$crime_property = ifelse(is.na(atl_bg3$crime_property), 0, atl_bg3$crime_property)


atl_bg3$crime_all = 
  atl_bg3$crime_violent + atl_bg3$crime_property


# add in com_density and phys_bound vars
load("atl_fin_pb3.RData")

# get indicator of physical boundaries by block
atl_phys = atl_fin_pb3 %>% dplyr::select(c(BLK_CODE, phys_bound))
atl_phys = atl_phys %>% as.data.frame() %>% dplyr::select(-c(geometry))

atl_bg3 <- merge(atl_bg3, atl_phys, by = "BLK_CODE")

# integrate commercial density
load("atl_cd.RData")
head(atl_cd)

# Calculate the average total jobs by census block
avg_total_jobs <- atl_cd %>%
  group_by(BLK_CODE) %>%
  summarise(total_jobs = mean(total_jobs, na.rm = TRUE))

atl_bg3 <- merge(atl_bg3, avg_total_jobs, by = "BLK_CODE")

# create com_density measure

atl_bg3$com_density <- atl_bg3$total_jobs/atl_bg3$pop
summary(atl_bg3$com_density)

#### quick edits before saving #### 

# resolving infinite numbers 

atl_bg3 = 
  atl_bg3 %>% 
  mutate(p_race_black_blv = ifelse(p_race_black_blv == Inf | p_race_black_blv == -Inf, NA, p_race_black_blv),
         p_race_hisp_blv = ifelse(p_race_hisp_blv == Inf | p_race_hisp_blv == -Inf, NA, p_race_hisp_blv),
         p_race_asian_blv = ifelse(p_race_asian_blv == Inf | p_race_asian_blv == -Inf, NA, p_race_asian_blv),
         p_race_white_blv = ifelse(p_race_white_blv == Inf | p_race_white_blv == -Inf, NA, p_race_white_blv))

# global measure

atl_bg3$edge_race_areal = 
  atl_bg3[, c("p_race_black_blv", "p_race_hisp_blv",
              "p_race_asian_blv", "p_race_white_blv")] %>% 
  mutate(geometry = NULL) %>% 
  as.data.frame %>% 
  as.matrix %>% 
  matrixStats::rowMaxs(x = .)

# pairwise measure 

atl_bg3 = 
  atl_bg3 %>% 
  mutate(edge_race_wb = p_race_black_blv * p_race_white_blv,
         edge_race_wh = p_race_hisp_blv * p_race_white_blv,
         edge_race_hb = p_race_hisp_blv * p_race_black_blv)

# arrest rate

atl_bg3$arrest_rate = (atl_bg3$total_arrests / atl_bg3$pop) * 10
atl_bg3$arrest_rate = 
  ifelse(atl_bg3$arrest_rate == Inf | atl_bg3$arrest_rate == -Inf, NA,
         atl_bg3$arrest_rate)
atl_bg3$lpop = log(atl_bg3$pop)

atl_bg3$felony_arrest_rate = (atl_bg3$felony_arrests / atl_bg3$pop) * 10
atl_bg3$felony_arrest_rate = 
  ifelse(atl_bg3$felony_arrest_rate == Inf | atl_bg3$felony_arrest_rate == -Inf, NA,
         atl_bg3$felony_arrest_rate)

atl_bg3$misdemeanor_arrest_rate = (atl_bg3$misdemeanor_arrests / atl_bg3$pop) * 10
atl_bg3$misdemeanor_arrest_rate = 
  ifelse(atl_bg3$misdemeanor_arrest_rate == Inf | atl_bg3$misdemeanor_arrest_rate == -Inf, NA,
         atl_bg3$misdemeanor_arrest_rate)

atl_bg3$violent_arrest_rate = (atl_bg3$violent_arrests / atl_bg3$pop) * 10
atl_bg3$violent_arrest_rate = 
  ifelse(atl_bg3$violent_arrest_rate == Inf | atl_bg3$violent_arrest_rate == -Inf, NA,
         atl_bg3$violent_arrest_rate)

atl_bg3$nonviolent_arrest_rate = (atl_bg3$nonviolent_arrests / atl_bg3$pop) * 10
atl_bg3$nonviolent_arrest_rate = 
  ifelse(atl_bg3$nonviolent_arrest_rate == Inf | atl_bg3$nonviolent_arrest_rate == -Inf, NA,
         atl_bg3$nonviolent_arrest_rate)

atl_bg3$society_arrest_rate = (atl_bg3$society_arrests / atl_bg3$pop) * 10
atl_bg3$society_arrest_rate = 
  ifelse(atl_bg3$society_arrest_rate == Inf | atl_bg3$society_arrest_rate == -Inf, NA,
         atl_bg3$society_arrest_rate)

atl_bg3$person_arrest_rate = (atl_bg3$person_arrests / atl_bg3$pop) * 10
atl_bg3$person_arrest_rate = 
  ifelse(atl_bg3$person_arrest_rate == Inf | atl_bg3$person_arrest_